# pip install pandas matplotlib bs4 wordcloud seaborn spacy nltlk install gensim pyLDAvis numpy re string 

import pandas as pd

import numpy as np

import re

import string

import unicodedata

​

"""data visualization"""

​

import matplotlib.pyplot as plt

from os import path

from PIL import Image

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

import plotly.express as px

import seaborn as sns

from matplotlib import rc

​

"""to supress warning"""

​

import warnings

warnings.filterwarnings("ignore",category=DeprecationWarning)

​

"""text cleaning"""

​

from bs4 import BeautifulSoup

​

"""stopwords, tokenizer, stemmer"""

​

​

​

import spacy

import nltk  

from nltk.corpus import stopwords

from nltk.tokenize import word_tokenize

from nltk.stem import PorterStemmer

from nltk.probability import FreqDist

from nltk.sentiment.vader import SentimentIntensityAnalyzer

​

from nltk.stem import WordNetLemmatizer 

from nltk.corpus import wordnet

​

#nltk.download('punkt')

#nltk.download('wordnet')

​

"""additional processing"""

​

import gensim

import gensim.corpora as corpora

from gensim.parsing.preprocessing import STOPWORDS

from gensim.parsing.preprocessing import remove_stopwords

from gensim.corpora import Dictionary 

from gensim.models.ldamodel import LdaModel

from gensim.models import CoherenceModel

from gensim.utils import simple_preprocess 

​

from collections import Counter

​

"""ML modelling"""

​

# Enable logging for gensim - optional

#import logging

#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)

​

#conda install -c conda-forge pyldavis

from sklearn.decomposition import LatentDirichletAllocation

​

#pip install flair

​

import pyLDAvis

​

from matplotlib import gridspec

import math

#pip install pyLDAvis.sklearn

import pyLDAvis.sklearn

​

from sklearn.feature_extraction.text import CountVectorizer

​

from nltk.stem import WordNetLemmatizer

from nltk.corpus import wordnet

#nltk.download('averaged_perceptron_tagger')

import warnings

warnings.filterwarnings("ignore", category=DeprecationWarning)

global_df = pd.read_csv('global_df.csv')

​

global_df.head(5)

global_df.shape

global_df = global_df.iloc[global_df.astype(str).drop_duplicates().index]

global_df.shape

# getting rows with topic/post content

​

global_df = global_df[global_df.cooked != '0']

global_df.shape

global_df['creation_date'] = pd.to_datetime(global_df['created_at'], format='%Y-%m-%d %H:%M:%S.%f') 

global_df['creation_year'] = global_df['creation_date'].dt.year

# Remove html code <p>

def remove_html(text):

    pattern = re.compile('<[^<]+?>')

    text = re.sub(pattern, ' ', text)

    return text

global_df['noHTML_text'] = global_df.cooked.apply(lambda x: remove_html(x))

global_df.to_csv('global_df.csv', header = True, encoding='utf-8', index=False)

# Function to clean text

​

def clean_text(text):

    # Remove html code <p>

    pattern = re.compile('<[^<]+?>')

    text = re.sub(pattern, ' ', text)

    # Remove hyperlinks

    text = re.sub(r'https?:\/\/\S+','',text)

    # lowercase

    text =text.lower()

​

    # Remove ticks, apostrophes and letters after " 's "

    text = re.sub("\’\w+", '', text)

    text = re.sub("\'\w+", '', text)

    # Remove all types of nums

    text = re.sub(r'\b\d+(?:\.\d+)?\s+', '', text)

    text = re.sub(r"\b\d+\b", ' ', text)

    # Remove specific punctuations

    text = re.sub('[^A-Za-z0-9]+', ' ', text)

    # Remove accents "é"

    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')

    # remove the hashtag symbol #

    text = re.sub(r'#','',text)

   # text = re.sub(r'[-./]', ' ', text)

    return text

global_df['cleaned_text'] = global_df.cooked.apply(lambda x: clean_text(x))

​

# Compare between Original Text & Clean Text

​

pd.options.display.max_colwidth = 300

global_df[['cooked', 'cleaned_text']].head(1)

# to avoid OS error run following command

# !python3 -m spacy download en_core_web_sm

nltk.download('stopwords')

# Holistic stop words function

​

nlp = spacy.load('en_core_web_sm')

​

set1 = list(nlp.Defaults.stop_words)

set2 = list(STOPWORDS)

set3 = list(stopwords.words('english'))

​

extra_set = ['ill', 'hi', 'im', 'ive', 'dont', 'hello', 'hey', 'like', 'thanks', 'maybe', 'q', 'e g', 'think', 'good', 'com', 'hop',

             'thats', 'c', 'b', 'l', 'il', 'x', 'z', 'v', 'f', 'e', 'q', 'isnt', 'wont', 'yes', 'want', 'let', 'know', 'id', 'g', 'thing',

             'thank', 'come', 'mm', 'w', 'non', 'day', 'look', 'baby', 'n', 'lot', 'way', 'use', 'try', 'hour', 'couple', 'week', 'ago',

             'i', 'h', 'et', 'img', 'cool', 'year', 'u', 'need', 'nice', 'guy', 'boy', 'pm', '3rd', 'pretty', 'sure', 'bit', 'week', 'minute',

             'png', 'screen', 'shot', 'sans', 'serif', 'mon', 'mar', 'kb', 'gmt', 'arial', 'helvetica', 'feb', 'long', 'period', 'time', 'font', 

             'size', 'medium', 'interested', 'dat', 'drank', 'dad', 'bottle', 'turn', 'number', 'everyday', 'follow', 'life', 'jar', 'j','www',

             'happen', 'yeah', 'wish', 'love', 'cheer', 'fun', 'later', 'bad', 'unfortunately', 'org', 'july', 'ki', 'february', 'month', 'saw',

             'potentially', '21st', 'hf', 'rr', 'lf', 'wait', 'yay', 'especially', 'feel', 'go']

​

stop_words_list = set1 + set2 + set3 + extra_set

​

​

# remove stopwords

def remove_stopwords(text): 

    slist = [word for word in text.split() if word not in stop_words_list]

    text = (" ").join(slist)

    return text

global_df['cleaned_text'] = global_df.cleaned_text.apply(lambda x: remove_stopwords(x))

# Function To Lemmatize Words To Its Root Form

​

lemmatizer = WordNetLemmatizer()

​

# function to convert nltk tag to wordnet tag

def nltk_tag_to_wordnet_tag(nltk_tag):

    if nltk_tag.startswith('J'):

        return wordnet.ADJ

    elif nltk_tag.startswith('V'):

        return wordnet.VERB

    elif nltk_tag.startswith('N'):

        return wordnet.NOUN

    elif nltk_tag.startswith('R'):

        return wordnet.ADV

    else:          

        return None

# Lemmatization function

def lemmatize_sentence(sentence):

    #tokenize the sentence and find the POS tag for each token

    nltk_tagged = nltk.pos_tag(nltk.word_tokenize(sentence))  

    #tuple of (token, wordnet_tag)

    wordnet_tagged = map(lambda x: (x[0], nltk_tag_to_wordnet_tag(x[1])), nltk_tagged)

    lemmatized_sentence = []

    for word, tag in wordnet_tagged:

        if tag is None:

            #if there is no available tag, append the token as is

            lemmatized_sentence.append(word)

        else:        

            #else use the tag to lemmatize the token

            lemmatized_sentence.append(lemmatizer.lemmatize(word, tag))

    return " ".join(lemmatized_sentence)

# to avoid errors with nltk library download the following 

nltk.download('punkt')

nltk.download('wordnet')

global_df['lemmat_text'] = global_df.cleaned_text.apply(lambda x: lemmatize_sentence(x))

global_df['lemmat_text'] = global_df.lemmat_text.apply(lambda x: remove_stopwords(x))

# Function To Lemmatize Words To Its Root Form and keep only NOUN 'NN' POS tag

​

lemmatizer = WordNetLemmatizer()

​

# function to convert nltk tag to wordnet tag

def nltk_tag_to_n_tag(nltk_tag):

    if nltk_tag.startswith('N'):

        return wordnet.NOUN 

    else:          

        return None

# Lemmatization function

def lemma_sentence(sentence):

    #tokenize the sentence and find the POS tag for each token

    nltk_tagged = nltk.pos_tag(nltk.word_tokenize(sentence))

    #tuple of (token, wordnet_tag)

    wordnet_tagged = map(lambda x: (x[0], nltk_tag_to_n_tag(x[1])), nltk_tagged)

    lemmatized_sentence = []

    for word, tag in wordnet_tagged:

        if tag is None:

            pass

        else: 

            lemmatized_sentence.append(lemmatizer.lemmatize(word, tag))

    return " ".join(lemmatized_sentence)

global_df['lemma_LDA_text'] = global_df.cleaned_text.apply(lambda x: lemma_sentence(x))

global_df['lemma_LDA_text'] = global_df.lemma_LDA_text.apply(lambda x: remove_stopwords(x))

# Tokenize Text 

def tokenize_text(text):

    filtered_text = []

    words = word_tokenize(text) 

​

    for word in words:

        filtered_text.append(word)

    return filtered_text

# tokenize all POS text

global_df['token_text'] = global_df.lemmat_text.apply(lambda x: tokenize_text(x))

# tokenize NOUN only POS text

global_df['token_NN_text'] = global_df.lemma_LDA_text.apply(lambda x: tokenize_text(x))

global_df[['lemmat_text', 'lemma_LDA_text']].head()

global_df.to_csv('global_df.csv', header = True, encoding='utf-8', index=False)

global_df.columns

mylist = []

for string in global_df['lemmat_text']:

    mylist.append(string)

mylist

​

new_text = str(mylist)

#tokenization

txt_tokens = word_tokenize(new_text)

words=[word.lower() for word in txt_tokens if word.isalpha()]

# Frequency Distribution Plot

fdist = FreqDist(words)

df_fdist = pd.DataFrame(fdist.items(), columns=['word', 'frequency'])

​

#top 30 words 

df_fdist30 = df_fdist.sort_values(['frequency'], ascending=False).head(30)

#ploting top 20 words 

plt.figure(figsize=(20,10))

sns.catplot(y="word", x="frequency", kind="bar", palette="vlag", data=df_fdist30, height=8.27, aspect=11.7/8.27)

plt.title('TOP 30 frequent words which occurred in the posts/topics')

df_21 = global_df[(global_df['creation_year'] == 2021) | (global_df['creation_year'] == 2020)]

mylist = []

for string in df_21['cleaned_text']:

    mylist.append(string)

mylist

​

new_text = str(mylist)

#tokenization

txt_tokens = word_tokenize(new_text)

words_21 = [word.lower() for word in txt_tokens if word.isalpha()]

# Frequency Distribution Plot

fdist = FreqDist(words_21)

df_fdist = pd.DataFrame(fdist.items(), columns=['word', 'frequency'])

​

#top 30 words 

df_fdist30 = df_fdist.sort_values(['frequency'], ascending=False).head(30)

df_fdist30

# using list comprehension 

listToStr= ' '.join(map(str, words)) 

​

# Create and generate a word cloud image:

wordcloud = WordCloud(width=1600, height=800, max_font_size=200, max_words=100, colormap='vlag',

                      background_color="white", collocations=True).generate(listToStr)

​

# Display the generated image:

plt.figure(figsize=(20,10))

plt.imshow(wordcloud)

plt.title('Wordcloud of TOP 100 frequent words for all posts/topics')

plt.axis("off")

plt.show()

# using list comprehension 

QS_word = ' '.join(map(str, words))

​

# open the image and use np.array to transform the file to an array

​

​

cand_mask= np.array(Image.open('QS.png'))

​

# take all values greater than 3 and transfer 255 (white)

# if they are less than 3, they will be whatever value they are in the array

cand_mask= np.where(cand_mask > 3, 255, cand_mask)

​

​

#create and generate our wordcloud object

wordcloud = WordCloud(background_color='white',

                      contour_color='blue',

                      mask=cand_mask,

                      contour_width=2).generate(QS_word)

​

​

#plot

plt.figure(figsize=(20,10))

plt.imshow(wordcloud, interpolation='bilinear')

plt.title('Wordcloud of TOP frequent words for Users posts/topics')

plt.axis('off')

plt.show()

#top 20 bigrams 

bigrams_series = (pd.Series(nltk.ngrams(words, 2)).value_counts())[:20]

bigrams_top = pd.DataFrame(bigrams_series.sort_values(ascending=False))

bigrams_top = bigrams_top.reset_index().rename(columns={'index': 'bigrams', 0:'counts'})

bigrams_top

plt.figure(figsize=(20,10))

sns.catplot(x = 'counts' , y='bigrams', kind="bar", palette="vlag", data=bigrams_top, height=8.27, aspect=11.7/8.27)

plt.title('TOP 20 pair words which occurred in the topics/posts')

#top 20 trigrams 

trigrams_series = (pd.Series(nltk.ngrams(words, 3)).value_counts())[:20]

trigrams_top = pd.DataFrame(trigrams_series.sort_values(ascending=False))

trigrams_top = trigrams_top.reset_index().rename(columns={'index': 'trigrams', 0:'counts'})

trigrams_top

plt.figure(figsize=(20,10))

sns.catplot(x = 'counts' , y='trigrams', kind="bar", palette="vlag", data=trigrams_top, height=8.27, aspect=11.7/8.27)

plt.title('TOP 20 three words which occurred together in the topics/posts')

test = global_df.copy()

# calculating the most frequent words' occurence in each thread 

​

test['data'] = test['no_sw_text'].str.count('data')

test['track'] = test['no_sw_text'].str.count('track')

test['sleep'] = test['no_sw_text'].str.count('sleep')

test['work'] = test['no_sw_text'].str.count('work')

test['app'] = test['no_sw_text'].str.count('app')

test['device'] = test['no_sw_text'].str.count('device')

test['self'] = test['no_sw_text'].str.count('self')

test['people'] = test['no_sw_text'].str.count('people')

test['help'] = test['no_sw_text'].str.count('help')

test['test'] = test['no_sw_text'].str.count('test')

test['start'] = test['no_sw_text'].str.count('start')

test['health'] = test['no_sw_text'].str.count('health')

test['measure'] = test['no_sw_text'].str.count('measure')

test['zeo'] = test['no_sw_text'].str.count('zeo')

test['post'] = test['no_sw_text'].str.count('post')

test['new'] = test['no_sw_text'].str.count('new')

test['question'] = test['no_sw_text'].str.count('question')

test['rate'] = test['no_sw_text'].str.count('rate')

test['user'] = test['no_sw_text'].str.count('user')

test['heart'] = test['no_sw_text'].str.count('heart')

test['change'] = test['no_sw_text'].str.count('change')

test['project'] = test['no_sw_text'].str.count('project')

test['sensor'] = test['no_sw_text'].str.count('sensor')

test['tool'] = test['no_sw_text'].str.count('tool')

test['different'] = test['no_sw_text'].str.count('different')

test['activity'] = test['no_sw_text'].str.count('activity')

test['idea'] = test['no_sw_text'].str.count('idea')

test['apps'] = test['no_sw_text'].str.count('apps')

test['record'] = test['no_sw_text'].str.count('record')

test['research'] = test['no_sw_text'].str.count('research')

words_df = test.groupby('creation_year')['data', 'track', 'sleep', 'work', 'app', 'device',

                             'self', 'people', 'help', 'test','start', 'health', 

                             'measure', 'zeo', 'post', 'new', 'question', 'rate',

                             'user', 'heart', 'change', 'project', 'sensor', 'tool', 'different',

                             'activity', 'idea', 'apps', 'record', 'research'].sum()

words_df

words_df.columns[0:30]

import plotly.graph_objs as go

​

fig = px.line(words_df, x=words_df.index, y=words_df.columns[0:30])

​

fig.show()

df = global_df.copy()

​

df['no_sw_LDA_text'] = df['no_sw_LDA_text'].astype('str')

df = df.groupby(['topic_id'], as_index = False).agg({'no_sw_LDA_text': ' '.join})

df.head()

df['token_NN_text'] = df.no_sw_LDA_text.apply(lambda x: tokenize_text(x))

dictionary = corpora.Dictionary(df.token_NN_text)

​

# filter words appear less than 15 docs & more than 0.5 documents & keep only 100k most frequent words

dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)

​

# document to 'bow' BAG OF WORDS

corpus = [dictionary.doc2bow(doc) for doc in df.token_NN_text]

len(corpus)

len(dictionary)

dictionary

%%time

# Creating the object for LDA model using gensim library

​

LDA = gensim.models.ldamodel.LdaModel

​

# Build LDA model

​

lda_model = LDA(corpus= corpus, 

                id2word = dictionary, 

                num_topics = 30, 

                random_state = 50,

                update_every = 1,

                chunksize = 100,

                passes = 10,

                #alpha = "auto"

               )

​

# save model

​

lda_model.save('QS_topics_lda.model')

​

#  num_topics =10, random_state = 100, chunksize=1000, passes =50, iterations = 100)

from gensim import corpora, models

​

tfidf = models.TfidfModel(corpus)

corpus_tfidf = tfidf[corpus]

lda_model = gensim.models.LdaMulticore(corpus, num_topics=10, 

                                       id2word=dictionary, passes=2, workers=2)

for idx, topic in lda_model.print_topics(-1):

    print('Topic: {} \nWords: {}'.format(idx, topic))

lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=10,

                                             id2word=dictionary, passes=2, workers=4)

​

for idx, topic in lda_model_tfidf.print_topics(-1):

    print('Topic: {} Word: {}'.format(idx, topic))

for index, score in sorted(lda_model[corpus[1]], key=lambda tup: -1*tup[1]):

​

    print("\nScore: {}\t \nTopic: {}".format(score, lda_model.print_topic(index, 10)))

for index, score in sorted(lda_model_tfidf[corpus[1]], key=lambda tup: -1*tup[1]):

    print("\nScore: {}\t \nTopic: {}".format(score, lda_model_tfidf.print_topic(index, 10)))

unseen_document = 'How a Pentagon deal became an identity crisis for Google'

unseen_document = lemma_sentence(unseen_document)

unseen_document = tokenize_text(unseen_document)

unseen_document

lda_model

bow_vector = dictionary.doc2bow(unseen_document)

​

for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):

    print("Score: {}\t Topic: {}".format(score, lda_model.print_topic(index, 5)))